using System;
using HtmlAgilityPack;
using System.Net;
using System.Text;
using System.IO;
using System.Collections.Generic;
using MySql.Data.MySqlClient;

public class CityName
{
    public const string HangZhou = "hangzhou";
}


public class HouseCrawler
{
    public const string DEFAULT_URL = "http://{0}.uoko.com/room/";
    private int mMaxPageIndex = 16;

    public static string GetHtmlText(string url, Encoding encode)
    {
        string strHTML = "";
        WebClient myWebClient = new WebClient();
        Stream myStream = myWebClient.OpenRead(url);
        StreamReader sr = new StreamReader(myStream, encode);//注意编码
        strHTML = sr.ReadToEnd();
        myStream.Close();
        return strHTML;
    }

    private HtmlDocument GetDocObj(string url)
    {
        //下载网页源代码 
        var docText = GetHtmlText(url, Encoding.Default);
        //加载源代码，获取文档对象
        var doc = new HtmlDocument(); doc.LoadHtml(docText);
        return doc;
    }

    private HtmlDocument GetDocObjByText(string text)
    {
        //加载源代码，获取文档对象
        var doc = new HtmlDocument(); doc.LoadHtml(text);
        return doc;
    }

    public void CrawAllHouse()
    {
        var url = "http://hangzhou.uoko.com/room/by0/";
        mMaxPageIndex = GetMaxPageIndex(url);

        for (int i = 0; i < mMaxPageIndex; i++)
        {
            CrawPageHouse(i + 1);
        }
    }

    private int GetMaxPageIndex(string url)
    {
        var doc = GetDocObj(url);
        var ret = 1;

        var navBar = doc.DocumentNode.SelectSingleNode(@"//div[@class='pagenation']");
        if (navBar == null)
        {
            return ret;
        }

        var aArr = navBar.SelectNodes(@"a");
        if (aArr == null)
        {
            return ret;
        }

        foreach (var item in aArr)
        {
            if (item.InnerText == "尾页")
            {
                var hrefValue = item.GetAttributeValue("href", string.Empty);
                if (string.IsNullOrEmpty(hrefValue) == true)
                {
                    return ret;
                }

                var tempStrs = hrefValue.Split('/', StringSplitOptions.RemoveEmptyEntries);
                if (tempStrs.Length != 2)
                {
                    return ret;
                }

                int.TryParse(tempStrs[1].Substring(5), out ret);
                break;
            }
        }

        return ret;
    }

    private void CrawPageHouse(int page = 1)
    {
        Console.WriteLine(string.Format("开始第{0}页", page));
        var url = string.Format("http://hangzhou.uoko.com/room/by0pg{0}/", page);
        var doc = GetDocObj(url);

        var itemLst = doc.DocumentNode.SelectNodes(@"//li[@class='gallery-item']");
        foreach (var item in itemLst)
        {
            var linkA = item.SelectSingleNode(@"div[@class='item-img']/a[1]");
            var relativeLinkStr = linkA.GetAttributeValue("href", string.Empty);

            var locaInfo = item.SelectSingleNode(@"div[@class='item-info']/div[1]");
            var location = locaInfo.InnerText;

            var priceInfo = item.SelectSingleNode(@"div[@class='item-info']/div[2]/span[1]");
            var price = priceInfo.InnerText;

            Console.WriteLine(string.Format("link:{0},localtion:{1},price:{2}", relativeLinkStr, location, price));
        }
    }

    public static string GetWebClient(string url)
    {
        string strHTML = "";
        WebClient myWebClient = new WebClient();
        Stream myStream = myWebClient.OpenRead(url);
        Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
        var gb2312 = Encoding.GetEncoding("GB2312");
        StreamReader sr = new StreamReader(myStream, gb2312);//注意编码
        strHTML = sr.ReadToEnd();
        myStream.Close();
        return strHTML;
    }

    /// <summary>添加省级-地区-县市 的城市信息，注意 省会城市 标记5</summary>
    /// <param name="cityCode">省份代码</param>
    public static void ParsePageByArea(String cityCode)
    {
        //更加链接格式和省份代码构造URL
        String url = String.Format("http://www.tianqihoubao.com/lishi/{0}.htm", cityCode);
        //下载网页源代码 
        var docText = GetWebClient(url);
        //加载源代码，获取文档对象
        var doc = new HtmlDocument(); doc.LoadHtml(docText);
        //更加xpath获取总的对象，如果不为空，就继续选择dl标签
        var res = doc.DocumentNode.SelectSingleNode(@"/html[1]/body[1]/div[1]/div[6]/div[1]/div[1]/div[3]");
        if (res != null)
        {
            var list = res.SelectNodes(@"dl");//选择标签数组
            if (list.Count < 1) return;
            foreach (var item in list)
            {
                var dd = item.SelectSingleNode(@"dd").SelectNodes("a");
                foreach (var node in dd)
                {
                    var text = node.InnerText.Trim();
                    //拼音代码要从href属性中进行分割提取
                    var herf = node.Attributes["href"].Value.Trim().Split('/', '.');
                    Console.WriteLine("{0}:{1}", text, herf[herf.Length - 2]);
                }
            }
        }
    }

    /// <summary>采集单个城市单个月的历史天气数据</summary>
    /// <param name="cityCode">城市拼音代码</param>
    /// <param name="year">年份</param>
    /// <param name="month">月份</param>
    public static void ParsePageByCityMonth(String cityCode, Int32 year, Int32 month)
    {
        //更加拼音代码，月份信息构造URL
        String url = String.Format("http://www.tianqihoubao.com/lishi/{0}/month/{1}{2:D2}.html", cityCode, year, month);
        //获取该链接的源代码
        var docText = GetWebClient(url);
        using (var fs = new FileStream("log.txt", FileMode.OpenOrCreate))
        {
            var ws = new StreamWriter(fs);
            ws.Write(docText);
            ws.Close();
        }
        //加载源代码，获取页面结构对象
        var doc = new HtmlDocument(); doc.LoadHtml(docText);
        // var temp = doc.DocumentNode.SelectSingleNode(@"//div[@id='content']/table[1]");
        // if (temp != null)
        // {
        //     foreach (var item in temp.ChildNodes)
        //     {
        //         Console.WriteLine(item.Name);
        //     }
        // }
        // else
        // {
        //     Console.WriteLine("no this child");
        // }
        //更加Xpath获取表格对象
        var res = doc.DocumentNode.SelectSingleNode(@"//div[@id='content']/table[1]");
        if (res != null)
        {
            //获取所有行
            var list = res.SelectNodes(@"tr");
            list.RemoveAt(0);//移除第一行，是表头
                             // 遍历每一行，获取日期，以及天气状况等信息
            foreach (var item in list)
            {
                var dd = item.SelectNodes(@"td");
                //日期 -  - 气温 - 风力风向
                if (dd.Count != 4) continue;
                //获取当前行日期
                var date1 = dd[0].InnerText.Replace("\r\n", "").Replace(" ", "").Trim();
                //获取当前行天气状况
                var tq = dd[1].InnerText.Replace("\r\n", "").Replace(" ", "").Trim();
                //获取当前行气温
                var qw = dd[2].InnerText.Replace("\r\n", "").Replace(" ", "").Trim();
                //获取当前行风力风向
                var fx = dd[3].InnerText.Replace("\r\n", "").Replace(" ", "").Trim();
                //输出

                Console.WriteLine("{0}:{1},{2},{3}", date1, tq, qw, fx);
            }
        }
    }

    #region temp
    public void ParseQuyu(string cityName)
    {
        var text = string.Empty;
        HtmlDocument doc;
        using (var file = new FileStream(string.Format("{0}.txt", cityName), FileMode.Open))
        {
            var sw = new StreamReader(file);
            text = sw.ReadToEnd();
            sw.Close();
        }
        if (string.IsNullOrEmpty(text) == false)
        {
            var url = string.Format("http://{0}.uoko.com/room/", cityName);
            doc = GetDocObj(url);
        }
        else
        {
            doc = GetDocObjByText(text);
        }

        var row = doc.DocumentNode.SelectSingleNode(@"//div[@class='filter-tags']/div[1]/div[1]");

        var arr = row.SelectNodes(@"a");

        var newUrls = new List<string>();
        foreach (var item in arr)
        {
            var hrefValue = item.GetAttributeValue("href", string.Empty);
            if (string.IsNullOrEmpty(hrefValue) == false)
            {
                // Console.WriteLine(hrefValue);
                var sectionArr = hrefValue.Split('/', StringSplitOptions.RemoveEmptyEntries);
                if (sectionArr.Length == 2)
                {
                    var newUrl = string.Format(DEFAULT_URL, CityName.HangZhou) + sectionArr[1] + "/";
                    newUrls.Add(newUrl);
                    Console.WriteLine(sectionArr[1]);
                }
            }
        }

        foreach (var item in newUrls)
        {
            ParseChildQuyu(item);
        }
    }

    private void ParseChildQuyu(string url)
    {
        var doc = GetDocObj(url);

        var row = doc.DocumentNode.SelectSingleNode(@"//div[@class='filter-tags']/div[1]/div[@class='filter-sub']");

        if (row == null)
        {
            return;
        }
        var arr = row.SelectNodes(@"a");

        var newUrls = new List<string>();
        if (arr != null)
        {
            foreach (var item in arr)
            {
                var hrefValue = item.GetAttributeValue("href", string.Empty);
                if (string.IsNullOrEmpty(hrefValue) == false)
                {
                    // Console.WriteLine(hrefValue);
                    var sectionArr = hrefValue.Split('/', StringSplitOptions.RemoveEmptyEntries);
                    if (sectionArr.Length == 2)
                    {
                        var newUrl = string.Format(DEFAULT_URL, CityName.HangZhou) + sectionArr[1] + "/";
                        newUrls.Add(newUrl);
                        Console.WriteLine(sectionArr[1]);
                    }
                }
            }
        }

    }

    public void DownloadHtml(string url, string fileName)
    {
        var str = GetHtmlText(url, Encoding.Default);
        using (var file = new FileStream(string.Format("{0}.txt", fileName), FileMode.OpenOrCreate))
        {
            var sw = new StreamWriter(file);
            sw.Write(str);
            sw.Close();
        }
    }
    #endregion





}